# -*- coding: utf-8 -*-
from scrapy import Spider,FormRequest,Request
from weibosearch.items import WeiboItem
import time
import re



class WeiboSpider(Spider):
    name = 'weibo'
    allowed_domains = ['weibo.cn']
    serach_url = 'https://weibo.cn/search/mblog'
    maxPage=2

    def start_requests(self):
        keyword='胡歌'
        url='{url}?keyword={keyword}'.format(url=self.serach_url,keyword=keyword)
        print('url: ',url)
        for page in range(self.maxPage+1):
            data={
                'mp':str(self.maxPage),
                'page':str(page)
            }
            yield FormRequest(url,callback=self.parse_index,formdata=data)
            time.sleep(3)
            

    def parse_index(self, response):
       weibos=response.css('div[id*="M_"]')
       for weibo in weibos:
            isForward=bool(weibo.css('span[class*="cmt"]').extract_first())
            print(isForward)
            if isForward:
                detailUrl=weibo.xpath('.//a[contains(.,"原文评论")]//@href').extract_first()
            else:
                detailUrl=weibo.xpath('.//a[contains(.,"评论")]//@href').extract_first()
            print("url",detailUrl)
            yield Request(detailUrl,callback=self.parse_detail)

    def parse_detail(self,response):
        id=re.search('comment\/(.*?)\?',response.url).group(1)
        url=response.url
        content=''.join(response.xpath('//div[@id="M_"]//span[@class="ctt"]//text()').extract())
        print(id,url,content)
        comment_count=response.xpath('//span[@class="pms"]//text()').re_first('评论\[(.*?)\]')
        forward_count=response.xpath('//a[contains(.,"转发[")]//text()').re_first('转发\[(.*?)\]')
        like_count=response.xpath('//a[contains(.,"赞[")]//text()').re_first('赞\[(.*?)\]')
        # print(comment_count,forward_count,like_count)
        posted_at=response.xpath('//div[@id="M_"]//span[@class="ct"]//text()').extract_first(default=None)
        user=response.xpath('//div[@id="M_"]/div[1]/a/text()').extract_first(default=None)
        # print(posted_at,user)
        weibo_item=WeiboItem()
        for field in weibo_item.fields:
            try:
                weibo_item[field]=eval(field)
            except NameError:
                self.logger.debug('field is not define'+field)
        yield weibo_item